import requests, telnetlib
import sys, Queue, threading
import time
import re

import dbmgr, getliveip
import addrutil

DEBUG_EYE = False
LG_SLEEP_SECONDS = 2

timeout_eye = Queue.Queue()
no_result_eye = Queue.Queue()
requests.defaults.defaults['pool_maxsize'] = 1
requests.defaults.defaults['pool_connections'] = 1
#requests.defaults.defaults['max_redirects'] = 0

class ProbeTargetThread(threading.Thread):
  def __init__ (self, eye, db, cmd_type, pfx, req_session=None):
    threading.Thread.__init__(self)
    # eye
    self.eye_id = eye.get('eye_id')
    self.counter_id = eye.get('counter_id')
    self.type = eye.get('type')
    self.addr = eye.get('addr').encode('ascii','ignore')
    self.user = eye.get('user').encode('ascii','ignore')
    self.pwd = eye.get('pwd').encode('ascii','ignore')
    self.promt = '\n' + eye.get('promt').encode('ascii','ignore')
    self.asn = eye.get('asn')
    self.ping_timeout = eye.get('ping_timeout')
    self.parse_ping = eye.get('parse_ping')
    self.parse_bgp = eye.get('parse_bgp')
    if not pfx.is_ipv6:
      self.ping_cmd = eye.get('ping4').encode('ascii','ignore')
      self.bgp_cmd = eye.get('bgp4').encode('ascii','ignore')
      self.bgpip = eye.get('bgp4ip')
    else:
      self.ping_cmd = eye.get('ping6').encode('ascii','ignore')
      self.bgp_cmd = eye.get('bgp6').encode('ascii','ignore')
      self.bgpip = eye.get('bgp6ip')
    self.req_session = req_session
    # db, pfx, time
    self.cmd_type = cmd_type
    self.db = db
    self.pfx = pfx
    self.deadline = pfx.bgptime + 120
    
  def run(self):
    if self.type == 'r':
      self.run_route_server()
    else:
      self.run_looking_glass()

  def get_cisco_best_aspath(self, bgp_output_lines):
    bestrt = ''
    ibst = 0
    for irl in xrange(len(bgp_output_lines)):
      rline = bgp_output_lines[irl]
      if ibst > 0:
        if re.match('  [\(1-9]', rline):
          ibst -= 1
          if ibst == 0:
            # , (123,456)
            # (123, 456)
            # {123, 456}
            bestrt = re.sub('[, ]*\([^\(]+\)[, ]*', ' ', rline)
            bestrt = re.sub('[, ]*\{[^\{]+\}[, ]*', ' ', bestrt).strip()
            break
        elif rline.startswith('  local'):
          ibst -= 1
          if ibst == 0:
            bestrt = '%d' % self.asn
            break
      elif rline.startswith('Paths: ('):
        firstline = rline.split(' ')
        for ifl in xrange(len(firstline)):
          if firstline[ifl] == 'best' and ifl+1<len(firstline):
            sbst = firstline[ifl+1][1:-1]
            if sbst.isdigit():
              ibst = int(sbst)
            break
    return bestrt

  def get_juniper_best_aspath(self, bgp_output_lines):
    bestrt = ''
    for irl in xrange(len(bgp_output_lines)):
      if re.search(' [\*\+]\[BGP', bgp_output_lines[irl]):
        if (irl+1 < len(bgp_output_lines) and
            bgp_output_lines[irl+1].startswith('---(more')):
          irl += 1
        if (irl+1 < len(bgp_output_lines) and
            re.search(' AS path: ', bgp_output_lines[irl+1])):
          # , (123,456)
          # (123, 456)
          # {123, 456}
          # AS path: 
          # I E ?
          bestrt = re.sub('[, ]*\([^\(]+\)[, ]*', ' ', bgp_output_lines[irl+1])
          bestrt = re.sub('[, ]*\{[^\{]+\}[, ]*', ' ', bestrt)
          bestrt = re.sub('.*AS path: ', '', bestrt)
          bestrt = re.sub('[^0-9]+$', '', bestrt).strip()
          if (not bestrt) or (bestrt.lower() == 'local'):
            bestrt = '%d' % self.asn
        break
    return bestrt

  def get_ping_res_dot_mark(self, ping_output, beg, end):
    n_resp = ping_output.count('!')
    if n_resp:
      delta = (end - beg - self.ping_timeout * (len(ping_output) - n_resp)
          ) / n_resp
    time_resp = {}
    elapse = beg
    for i in xrange(len(ping_output)):
      if ping_output[i] == '!':
        time_resp[int(elapse)] = 1
        elapse += delta
      else:
        time_resp[int(elapse)] = 0
        elapse += self.ping_timeout
    return time_resp

  def parse_bgp_detail(self, commander):
    prev_is_good_path = -1
    if not self.bgpip:
      bgp_cmd = "%s%s\n" % (self.bgp_cmd, self.pfx.dec)
    else:
      bgp_cmd = "%s%s\n" % (self.bgp_cmd, self.pfx.liveip)
    bgp_cmd_exp = '%s%s' % (self.promt, bgp_cmd[:-1])
    promt_more_exp_re = re.compile('[^\x08]+\n  .+\n --More-- ', re.DOTALL)
    promt_exp_re = re.compile(
        '[^\x08]+\n  .+%s' % self.promt.replace('.', '\.'), re.DOTALL)
    if DEBUG_EYE:
      print >> sys.stderr, "\n>>>>>>>>>> begin bgp [bgp_cmd]"
      nlp = 0 
      print >> sys.stderr, bgp_cmd
      print >> sys.stderr, "\n---------- [bgp_cmd_exp]"
      print >> sys.stderr, bgp_cmd_exp
      print >> sys.stderr, "\n<<<<<<<<<<"
    while True:
      if time.time() > self.deadline:
        return
      # get AS-path
      commander.write('\n' + bgp_cmd)
      bgp_cmd_before = commander.read_until(bgp_cmd_exp, 15)
      if DEBUG_EYE:
        print >> sys.stderr, "\n>>>>>>>>>> send bgp cmd [bgp_cmd_before]"
        print >> sys.stderr, bgp_cmd_before
        print >> sys.stderr, "\n<<<<<<<<<<"
      if not bgp_cmd_before.endswith(bgp_cmd_exp):
        return
      beg = time.time()
      reslines = []
      while True:
        exp_res = commander.expect([promt_more_exp_re, promt_exp_re], 15)
        if DEBUG_EYE:
          print >> sys.stderr, "\n>>>>>>>>>> get bgp output [exp_res]"
          print >> sys.stderr, exp_res
          print >> sys.stderr, "\n<<<<<<<<<<"
        if exp_res[0] == 0: # --More--
          commander.write(' ')
          reslines.extend(exp_res[2].split('\r\n'))
        elif exp_res[0] == 1:
          reslines.extend(exp_res[2].split('\r\n'))
          break
        else:
          return
        if time.time() > self.deadline:
          return
      end = time.time()
      bgp_res_time = int((beg + end) / 2)
      if DEBUG_EYE:
        print >> sys.stderr, "\n>>>>>>>>>> get route [route]"
        for l in reslines:
          print >> sys.stderr, l
        print >> sys.stderr, "\n---------- [beg, end, bgp_res_time]"
        print >> sys.stderr, beg, end, bgp_res_time
        print >> sys.stderr, "\n<<<<<<<<<<"
      # get best route
      bestrt = self.get_cisco_best_aspath(reslines)
      if DEBUG_EYE:
        print >> sys.stderr, "\n>>>>>>>>>> get best route [bestrt]"
        print >> sys.stderr, '[%r]' % bestrt
        print >> sys.stderr, "\n<<<<<<<<<<"
      if not bestrt:
        time.sleep(1)
        continue
      if re.search(self.pfx.bad_path_re, bestrt):
        is_good_path = 0 
      else:
        is_good_path = 1
      if is_good_path != -1 and prev_is_good_path != is_good_path:
        if not DEBUG_EYE:
          self.db.dbQueue.put(("route", (self.pfx.alarm_id, bgp_res_time,
            self.eye_id, is_good_path, self.pfx.bgptime)))
        prev_is_good_path = is_good_path 
      if DEBUG_EYE:
        print >> sys.stderr, "\n>>>>>>>>>> bgp over [nlp]"
        nlp += 1
        print >> sys.stderr, nlp
        if nlp >= 2:
          return
        print >> sys.stderr, "\n<<<<<<<<<<"
      time.sleep(1)

  def parse_bgp_table(self, commander):
    prev_is_good_path = -1
    if not self.bgpip:
      bgp_cmd = "%s%s\n" % (self.bgp_cmd, self.pfx.dec)
    else:
      bgp_cmd = "%s%s\n" % (self.bgp_cmd, self.pfx.liveip)
    bgp_cmd_exp = '%s%s' % (self.promt, bgp_cmd[:-1])
    promt_more_exp_re = re.compile(
        '.+\n  .+\n---\(more( [0-9]+\%)?\)---', re.DOTALL)
    promt_exp_re = re.compile(
        '.+\n  .+%s' % self.promt.replace('.', '\.'), re.DOTALL)
    if DEBUG_EYE:
      print >> sys.stderr, "\n>>>>>>>>>> begin bgp [bgp_cmd]"
      nlp = 0 
      print >> sys.stderr, bgp_cmd
      print >> sys.stderr, "\n---------- [bgp_cmd_exp]"
      print >> sys.stderr, bgp_cmd_exp
      print >> sys.stderr, "\n<<<<<<<<<<"
    while True:
      if time.time() > self.deadline:
        return
      # get AS-path
      commander.write('\n' + bgp_cmd)
      bgp_cmd_before = commander.read_until(bgp_cmd_exp, 15)
      if DEBUG_EYE:
        print >> sys.stderr, "\n>>>>>>>>>> send bgp cmd [bgp_cmd_before]"
        print >> sys.stderr, bgp_cmd_before
        print >> sys.stderr, "\n<<<<<<<<<<"
      if not bgp_cmd_before.endswith(bgp_cmd_exp):
        return
      beg = time.time()
      reslines = []
      while True:
        exp_res = commander.expect([promt_more_exp_re, promt_exp_re], 15)
        if DEBUG_EYE:
          print >> sys.stderr, "\n>>>>>>>>>> get bgp output [exp_res]"
          print >> sys.stderr, exp_res
          print >> sys.stderr, "\n<<<<<<<<<<"
        if exp_res[0] == 0: # --More--
          commander.write(' ')
          reslines.extend(exp_res[2].split('\r\n'))
        elif exp_res[0] == 1:
          reslines.extend(exp_res[2].split('\r\n'))
          break
        else:
          return
        if time.time() > self.deadline:
          return
      end = time.time()
      bgp_res_time = int((beg + end) / 2)
      if DEBUG_EYE:
        print >> sys.stderr, "\n>>>>>>>>>> get route [route]"
        for l in reslines:
          print >> sys.stderr, l
        print >> sys.stderr, "\n---------- [beg, end, bgp_res_time]"
        print >> sys.stderr, beg, end, bgp_res_time
        print >> sys.stderr, "\n<<<<<<<<<<"
      # get best route
      bestrt = self.get_juniper_best_aspath(reslines)
      if DEBUG_EYE:
        print >> sys.stderr, "\n>>>>>>>>>> get best route [bestrt]"
        print >> sys.stderr, '[%r]' % bestrt
        print >> sys.stderr, "\n<<<<<<<<<<"
      if not bestrt:
        time.sleep(1)
        continue
      if re.search(self.pfx.bad_path_re, bestrt):
        is_good_path = 0 
      else:
        is_good_path = 1
      if is_good_path != -1 and prev_is_good_path != is_good_path:
        if not DEBUG_EYE:
          self.db.dbQueue.put(("route", (self.pfx.alarm_id, bgp_res_time,
            self.eye_id, is_good_path, self.pfx.bgptime)))
        prev_is_good_path = is_good_path 
      if DEBUG_EYE:
        print >> sys.stderr, "\n>>>>>>>>>> bgp over [nlp]"
        nlp += 1
        print >> sys.stderr, nlp
        if nlp >= 2:
          return
        print >> sys.stderr, "\n<<<<<<<<<<"
      time.sleep(1)

  def parse_ping_short(self, commander):
    presp = -1
    ptime = -1
    ping_cmd = "%s%s\n" % (self.ping_cmd, self.pfx.liveip)
    ping_exp_re = re.compile(
        '%s.+\r\n[!\S]{5}\r\n' % ping_cmd[:-1].replace('.', '\.'), re.DOTALL)
    addrutil.log(int(time.time()), 0, "Y:ping command: %s" % ping_cmd)
    if DEBUG_EYE:
      print >> sys.stderr, "\n>>>>>>>>>> begin ping [ping_cmd]"
      nlp = 0 
      print >> sys.stderr, ping_cmd
      print >> sys.stderr, "\n<<<<<<<<<<"
    while True:
      if time.time() > self.deadline:
        return
      commander.write(ping_cmd)
      beg = time.time()
      ping_exp = commander.expect([ping_exp_re, ], 5*self.ping_timeout+5)
      end = time.time()
      if ping_exp[0] == 0:
        resp_str = ping_exp[2][-7:-2]
      else:
        return
      addrutil.log(int(time.time()), 0, "Y:ping output: %s | %s" % (ping_exp, resp_str))
      if DEBUG_EYE:
        print >> sys.stderr, "\n>>>>>>>>>> get ping output [ping_exp]"
        print >> sys.stderr, ping_exp
        print >> sys.stderr, "\n---------- [resp_str]"
        print >> sys.stderr, resp_str
        print >> sys.stderr, "\n<<<<<<<<<<"
      # get time_resp
      time_resp = self.get_ping_res_dot_mark(resp_str, beg, end)
      for ctime in xrange(int(beg), int(end)+1):
        if ctime in time_resp:
          resp = time_resp[ctime]
          if (presp != resp) and (ptime != ctime):
            if not DEBUG_EYE:
              self.db.dbQueue.put(("probe", (self.pfx.alarm_id, ctime,
                self.eye_id, resp, self.pfx.bgptime)))
            presp = resp
            ptime = ctime
      addrutil.log(int(time.time()), 0, "Y:ping result: %s" % time_resp)
      if DEBUG_EYE:
        print >> sys.stderr, "\n>>>>>>>>>> get ping result [time_resp]"
        print >> sys.stderr, time_resp
        print >> sys.stderr, "\n---------- [beg, end, nlp]"
        nlp += 1
        print >> sys.stderr, beg, end, nlp
        if nlp >= 2:
          return
        print >> sys.stderr, "\n<<<<<<<<<<"
      time.sleep(1)

  def parse_ping_loop(self, commander):
    presp = -1
    ptime = -1
    ping_cmd = "%s%s\n" % (self.ping_cmd, self.pfx.liveip)
    ping_res_read = 'time='
    addrutil.log(int(time.time()), 0, "Y:ping command: %s" % ping_cmd)
    if DEBUG_EYE:
      print >> sys.stderr, "\n>>>>>>>>>> begin ping [ping_cmd]"
      nlp = 0 
      print >> sys.stderr, ping_cmd
      print >> sys.stderr, "\n---------- [ping_res_read]"
      print >> sys.stderr, ping_res_read
      print >> sys.stderr, "\n<<<<<<<<<<"
    commander.write(ping_cmd)
    while True:
      ctime = int(time.time())
      if ctime > self.deadline:
        break
      ping_res = commander.read_until(ping_res_read, self.ping_timeout)
      if ping_res.endswith(ping_res_read):
        resp = 1
      elif ping_res:
        resp = 0
      else:
        return
      if (presp != resp) and (ptime != ctime):
        if not DEBUG_EYE:
          self.db.dbQueue.put(("probe", (self.pfx.alarm_id, ctime,
            self.eye_id, resp, self.pfx.bgptime)))
        presp = resp
        ptime = ctime
      addrutil.log(int(time.time()), 0, "Y:ping output: %s | %s | %s" % (ctime, resp, ping_res))
      if DEBUG_EYE:
        print >> sys.stderr, "\n>>>>>>>>>> get ping output [resp]"
        print >> sys.stderr, ctime, resp
        print >> sys.stderr, "\n---------- [ping_res]"
        print >> sys.stderr, ping_res
        print >> sys.stderr, "\n---------- [nlp]"
        nlp += 1
        print >> sys.stderr, nlp
        if nlp >= 2:
          return
        print >> sys.stderr, "\n<<<<<<<<<<"

  def get_url_param(self, urlparams):
    param = {}
    for kv in urlparams.split('&'):
      kv = kv.split('=')
      param[kv[0]] = kv[1]
    return param

  def parse_helg_bgp(self):
    prev_is_good_path = -1
    # session
    #headers = {'Connection': 'keep-alive', }
    bgp_req = self.req_session
    urlp = self.addr.split('&')
    if self.bgpip:
      urlparams = urlp[1] + self.bgp_cmd + self.pfx.liveip
    else:
      urlparams = urlp[1] + self.bgp_cmd + self.pfx.dec
    data = self.get_url_param(urlparams)
    if DEBUG_EYE:
      print >> sys.stderr, "\n>>>>>>>>>> begin bgp [urlp]"
      print >> sys.stderr, urlp[0], urlp[1], urlparams
      nlp = 0 
      print >> sys.stderr, "\n<<<<<<<<<<"
    # run
    while True:
      if time.time() > self.deadline:
        return
      bestrt = ''
      beg = time.time()
      #resp = bgp_req.post(urlp[0], headers=headers, data=data)
      post_resp = bgp_req.post(urlp[0], data=data, timeout=30)
      if post_resp and post_resp.text:
        bestrt_match = re.match(
            '.+lg_return.+Status: .?B.+AS_PATH: ([^\n]+)\n.+',
            post_resp.text, re.DOTALL)
        if bestrt_match:
          bestrt = re.sub('[, ]*\([^\(]+\)[, ]*', ' ', bestrt_match.group(1))
          bestrt = re.sub('[, ]*\{[^\{]+\}[, ]*', ' ', bestrt).strip()
          if bestrt.lower() == 'local':
            bestrt = '%d' % self.asn
        else:
          time.sleep(LG_SLEEP_SECONDS)
          continue
      else:
        return
      end = time.time()
      bgp_res_time = int((beg + end) / 2)
      if DEBUG_EYE:
        print >> sys.stderr, "\n>>>>>>>>>> get bgp output [bestrt]"
        print >> sys.stderr, bestrt
        print >> sys.stderr, "\n---------- [beg, end, bgp_res_time]"
        print >> sys.stderr, beg, end, bgp_res_time
        print >> sys.stderr, "\n<<<<<<<<<<"
      if not bestrt:
        time.sleep(LG_SLEEP_SECONDS)
        continue
      if re.search(self.pfx.bad_path_re, bestrt):
        is_good_path = 0
      else:
        is_good_path = 1
      if is_good_path != -1 and prev_is_good_path != is_good_path:
        if not DEBUG_EYE:
          self.db.dbQueue.put(("route", (self.pfx.alarm_id, bgp_res_time,
            self.eye_id, is_good_path, self.pfx.bgptime)))
        prev_is_good_path = is_good_path 
      if DEBUG_EYE:
        print >> sys.stderr, "\n>>>>>>>>>> bgp over [nlp]"
        nlp += 1
        print >> sys.stderr, nlp
        if nlp >= 2:
          return
        print >> sys.stderr, "\n<<<<<<<<<<"
      time.sleep(LG_SLEEP_SECONDS)

  def parse_helg_ping(self):
    presp = -1
    ptime = -1
    # session
    #headers = {'Connection': 'keep-alive', }
    ping_req = self.req_session
    urlp = self.addr.split('&')
    urlparams = urlp[1] + self.ping_cmd + self.pfx.liveip
    data = self.get_url_param(urlparams)
    if DEBUG_EYE:
      print >> sys.stderr, "\n>>>>>>>>>> begin ping [urlp]"
      print >> sys.stderr, urlp[0], urlp[1], urlparams
      nlp = 0 
      print >> sys.stderr, "\n<<<<<<<<<<"
    # run
    while True:
      if time.time() > self.deadline:
        return
      ping_res = ''
      resp_str = None
      beg = time.time()
      #resp = ping_req.post(urlp[0], headers=headers, data=data)
      post_resp = ping_req.post(urlp[0], data=data,
          timeout=self.ping_timeout*5+15)
      if post_resp and post_resp.text:
        ping_match = re.match(
            '.+lg_return.+<pre>(.+)</pre>', post_resp.text, re.DOTALL)
        if ping_match:
          ping_res = ping_match.group(1).split('\n')
          resp_str = []
          for l in ping_res:
            if l.startswith('Request timed out'):
              resp_str.append('.')
            elif l.startswith('Reply from '):
              if 'time=' in l:
                resp_str.append('!')
              else:
                resp_str.append('.')
          resp_str = ''.join(resp_str)
        else:
          return
      else:
        return
      end = time.time()
      if DEBUG_EYE:
        print >> sys.stderr, "\n>>>>>>>>>> get ping output [ping_res]"
        print >> sys.stderr, ping_res
        print >> sys.stderr, "\n---------- [resp_str]"
        print >> sys.stderr, resp_str
        print >> sys.stderr, "\n---------- [beg, end]"
        print >> sys.stderr, beg, end
        print >> sys.stderr, "\n<<<<<<<<<<"
      if not resp_str:
        return
      time_resp = self.get_ping_res_dot_mark(resp_str, beg, end)
      for ctime in xrange(int(beg), int(end)+1):
        if ctime in time_resp:
          resp = time_resp[ctime]
          if (presp != resp) and (ptime != ctime):
            if not DEBUG_EYE:
              self.db.dbQueue.put(("probe", (self.pfx.alarm_id, ctime,
                self.eye_id, resp, self.pfx.bgptime)))
            presp = resp
            ptime = ctime
      if DEBUG_EYE:
        print >> sys.stderr, "\n>>>>>>>>>> get ping result [time_resp]"
        print >> sys.stderr, time_resp
        print >> sys.stderr, "\n---------- [nlp]"
        nlp += 1
        print >> sys.stderr, nlp
        if nlp >= 2:
          return
        print >> sys.stderr, "\n<<<<<<<<<<"
      time.sleep(LG_SLEEP_SECONDS)

  def parse_level3lg_bgp(self):
    prev_is_good_path = -1
    # session
    bgp_req = self.req_session
    if self.bgpip:
      url = (self.addr.replace('CMD', 'bgp')
          + self.bgp_cmd + self.pfx.liveip)
    else:
      url = (self.addr.replace('CMD', 'bgp')
          + self.bgp_cmd + self.pfx.dec)
    if DEBUG_EYE:
      print >> sys.stderr, "\n>>>>>>>>>> begin bgp [url]"
      print >> sys.stderr, url
      nlp = 0 
      print >> sys.stderr, "\n<<<<<<<<<<"
    # run
    while True:
      if time.time() > self.deadline:
        return
      bestrt = ''
      beg = time.time()
      get_resp = bgp_req.get(url, timeout=30)
      if get_resp and get_resp.text:
        bestrt_match = re.match(
            '.+\nPaths: \(.+\n<font color=green>  ([^\n]+)\n.+',
            get_resp.text, re.DOTALL)
        if bestrt_match:
          bestrt = re.sub('[, ]*\([^\(]+\)[, ]*', ' ', bestrt_match.group(1))
          bestrt = re.sub('[, ]*\{[^\{]+\}[, ]*', ' ', bestrt).strip()
          if bestrt.lower() == 'local':
            bestrt = '%d' % self.asn
        else:
          time.sleep(LG_SLEEP_SECONDS)
          continue
      else:
        return
      end = time.time()
      bgp_res_time = int((beg + end) / 2)
      if DEBUG_EYE:
        print >> sys.stderr, "\n>>>>>>>>>> get bgp output [bestrt]"
        print >> sys.stderr, bestrt
        print >> sys.stderr, "\n---------- [beg, end, bgp_res_time]"
        print >> sys.stderr, beg, end, bgp_res_time
        print >> sys.stderr, "\n<<<<<<<<<<"
      if not bestrt:
        time.sleep(LG_SLEEP_SECONDS)
        continue
      if re.search(self.pfx.bad_path_re, bestrt):
        is_good_path = 0
      else:
        is_good_path = 1
      if is_good_path != -1 and prev_is_good_path != is_good_path:
        if not DEBUG_EYE:
          self.db.dbQueue.put(("route", (self.pfx.alarm_id, bgp_res_time,
            self.eye_id, is_good_path, self.pfx.bgptime)))
        prev_is_good_path = is_good_path 
      if DEBUG_EYE:
        print >> sys.stderr, "\n>>>>>>>>>> bgp over [nlp]"
        nlp += 1
        print >> sys.stderr, nlp
        if nlp >= 2:
          return
        print >> sys.stderr, "\n<<<<<<<<<<"
      time.sleep(LG_SLEEP_SECONDS)

  def parse_level3lg_ping(self):
    presp = -1
    ptime = -1
    # session
    ping_req = self.req_session
    url = (self.addr.replace('CMD', 'ping')
        + self.ping_cmd + self.pfx.liveip)
    if DEBUG_EYE:
      print >> sys.stderr, "\n>>>>>>>>>> begin ping [url]"
      print >> sys.stderr, url
      nlp = 0 
      print >> sys.stderr, "\n<<<<<<<<<<"
    # run
    while True:
      if time.time() > self.deadline:
        return
      resp = -1
      beg = time.time()
      get_resp = ping_req.get(url, timeout=self.ping_timeout+15)
      if get_resp and get_resp.text:
        ping_match = re.match(
            '.+<pre>(.+)</pre>', get_resp.text, re.DOTALL)
        if ping_match:
          if 'time=' in ping_match.group(1):
            resp = 1
          else:
            resp = 0
        else:
          return
      else:
        return
      end = time.time()
      ctime = int((beg + end) / 2)
      if DEBUG_EYE:
        print >> sys.stderr, "\n>>>>>>>>>> get ping output [resp]"
        print >> sys.stderr, resp
        print >> sys.stderr, "\n---------- [beg, end, ping_res_time]"
        print >> sys.stderr, beg, end, ctime
        print >> sys.stderr, "\n<<<<<<<<<<"
      if resp != -1 and (presp != resp) and (ptime != ctime):
        if not DEBUG_EYE:
          self.db.dbQueue.put(("probe", (self.pfx.alarm_id, ctime,
            self.eye_id, resp, self.pfx.bgptime)))
        presp = resp
        ptime = ctime
      if DEBUG_EYE:
        print >> sys.stderr, "\n>>>>>>>>>> get ping result [nlp]"
        nlp += 1
        print >> sys.stderr, nlp
        if nlp >= 2:
          return
        print >> sys.stderr, "\n<<<<<<<<<<"
      time.sleep(LG_SLEEP_SECONDS)

  def parse_mtulg_bgp(self):
    prev_is_good_path = -1
    router_type = (self.addr[self.addr.rindex(':')+1 : ] == 'cisco' and 1) or 0
    # session
    bgp_req = self.req_session
    urlp = self.addr.split('&')
    if self.bgpip:
      urlparams = urlp[1] + self.bgp_cmd + self.pfx.liveip
    else:
      urlparams = urlp[1] + self.bgp_cmd + self.pfx.dec
    data = self.get_url_param(urlparams)
    if DEBUG_EYE:
      print >> sys.stderr, "\n>>>>>>>>>> begin bgp [urlp_0, urlp_1, urlparams]"
      print >> sys.stderr, urlp[0], urlp[1], urlparams
      nlp = 0 
      print >> sys.stderr, "\n---------- [data]"
      print >> sys.stderr, data
      print >> sys.stderr, "\n<<<<<<<<<<"
    # run
    while True:
      if time.time() > self.deadline:
        return
      bestrt = ''
      beg = time.time()
      post_resp = bgp_req.post(urlp[0], data=data, timeout=30)
      if post_resp and post_resp.text:
        if router_type == 1: # cisco
          route_match = re.match('.+<pre>.*(\nBGP.+)</pre>',
              post_resp.text, re.DOTALL)
          if route_match:
            reslines = route_match.group(1).split('\n')
            bestrt = self.get_cisco_best_aspath(reslines)
          else:
            time.sleep(LG_SLEEP_SECONDS)
            continue
        else: # juniper
          route_match = re.match('.+<pre>(.+BGP.+)</pre>',
              post_resp.text, re.DOTALL)
          if route_match:
            reslines = route_match.group(1).split('\n')
            bestrt = self.get_juniper_best_aspath(reslines)
          else:
            time.sleep(LG_SLEEP_SECONDS)
            continue
      else:
        return
      end = time.time()
      bgp_res_time = int((beg + end) / 2)
      if DEBUG_EYE:
        print >> sys.stderr, "\n>>>>>>>>>> get bgp output [bestrt]"
        print >> sys.stderr, bestrt
        print >> sys.stderr, "\n---------- [html]"
        print >> sys.stderr, post_resp.text
        print >> sys.stderr, "\n---------- [beg, end, bgp_res_time]"
        print >> sys.stderr, beg, end, bgp_res_time
        print >> sys.stderr, "\n<<<<<<<<<<"
      if not bestrt:
        time.sleep(LG_SLEEP_SECONDS)
        continue
      if re.search(self.pfx.bad_path_re, bestrt):
        is_good_path = 0
      else:
        is_good_path = 1
      if is_good_path != -1 and prev_is_good_path != is_good_path:
        if not DEBUG_EYE:
          self.db.dbQueue.put(("route", (self.pfx.alarm_id, bgp_res_time,
            self.eye_id, is_good_path, self.pfx.bgptime)))
        prev_is_good_path = is_good_path 
      if DEBUG_EYE:
        print >> sys.stderr, "\n>>>>>>>>>> bgp over [nlp]"
        nlp += 1
        print >> sys.stderr, nlp
        if nlp >= 2:
          return
        print >> sys.stderr, "\n<<<<<<<<<<"
      time.sleep(LG_SLEEP_SECONDS)

  def parse_mtulg_ping(self):
    presp = -1
    ptime = -1
    # session
    ping_req = self.req_session
    urlp = self.addr.split('&')
    urlparams = urlp[1] + self.ping_cmd + self.pfx.liveip
    data = self.get_url_param(urlparams)
    if DEBUG_EYE:
      print >> sys.stderr, "\n>>>>>>>>>> begin ping [urlp]"
      print >> sys.stderr, urlp[0], urlp[1], urlparams
      nlp = 0 
      print >> sys.stderr, "\n<<<<<<<<<<"
    # run
    while True:
      if time.time() > self.deadline:
        return
      resp_str = ''
      beg = time.time()
      post_resp = ping_req.post(urlp[0], data=data,
          timeout=self.ping_timeout*5+15)
      if post_resp and post_resp.text:
        ping_match = re.match(
            '.+<pre>.+\n([!\S]{5})\n.+</pre>', post_resp.text, re.DOTALL)
        if ping_match:
          resp_str = ping_match.group(1)
        else:
          return
      else:
        return
      end = time.time()
      if DEBUG_EYE:
        print >> sys.stderr, "\n>>>>>>>>>> get ping output [resp_str]"
        print >> sys.stderr, resp_str
        print >> sys.stderr, "\n---------- [html]"
        print >> sys.stderr, post_resp.text
        print >> sys.stderr, "\n---------- [beg, end]"
        print >> sys.stderr, beg, end
        print >> sys.stderr, "\n<<<<<<<<<<"
      time_resp = self.get_ping_res_dot_mark(resp_str, beg, end)
      for ctime in xrange(int(beg), int(end)+1):
        if ctime in time_resp:
          resp = time_resp[ctime]
          if (presp != resp) and (ptime != ctime):
            if not DEBUG_EYE:
              self.db.dbQueue.put(("probe", (self.pfx.alarm_id, ctime,
                self.eye_id, resp, self.pfx.bgptime)))
            presp = resp
            ptime = ctime
      if DEBUG_EYE:
        print >> sys.stderr, "\n>>>>>>>>>> get ping result [time_resp]"
        print >> sys.stderr, time_resp
        print >> sys.stderr, "\n---------- [nlp]"
        nlp += 1
        print >> sys.stderr, nlp
        if nlp >= 2:
          return
        print >> sys.stderr, "\n<<<<<<<<<<"
      time.sleep(LG_SLEEP_SECONDS)

  def parse_gbxslg_bgp(self):
    prev_is_good_path = -1
    # session
    bgp_req = self.req_session
    urlp = self.addr.split('&')
    if self.bgpip:
      urlparams = urlp[1] + self.bgp_cmd + self.pfx.liveip
    else:
      urlparams = urlp[1] + self.bgp_cmd + self.pfx.dec
    data = self.get_url_param(urlparams)
    if DEBUG_EYE:
      print >> sys.stderr, "\n>>>>>>>>>> begin bgp [urlp]"
      print >> sys.stderr, urlp[0], urlp[1], urlparams
      nlp = 0 
      print >> sys.stderr, "\n<<<<<<<<<<"
    # run
    while True:
      if time.time() > self.deadline:
        return
      bestrt = ''
      beg = time.time()
      post_resp = bgp_req.post(urlp[0], data=data, timeout=30)
      if post_resp and post_resp.text:
        bestrt_match = re.match(
            '.+<PRE>.*\nBGP.+\n<FONT COLOR="#FF0000">  ([^ ][^<]+)</FONT>',
            post_resp.text, re.DOTALL)
        if bestrt_match:
          bestrt = re.sub('[, ]*\([^\(]+\)[, ]*', ' ', bestrt_match.group(1))
          bestrt = re.sub('[, ]*\{[^\{]+\}[, ]*', ' ', bestrt).strip()
          if bestrt.lower() == 'local':
            bestrt = '%d' % self.asn
        else:
          time.sleep(LG_SLEEP_SECONDS)
          continue
      else:
        return
      end = time.time()
      bgp_res_time = int((beg + end) / 2)
      if DEBUG_EYE:
        print >> sys.stderr, "\n>>>>>>>>>> get bgp output [bestrt]"
        print >> sys.stderr, bestrt
        #print >> sys.stderr, "\n---------- [html]"
        #print >> sys.stderr, post_resp.text
        print >> sys.stderr, "\n---------- [beg, end, bgp_res_time]"
        print >> sys.stderr, beg, end, bgp_res_time
        print >> sys.stderr, "\n<<<<<<<<<<"
      if not bestrt:
        is_good_path = -1
      if re.search(self.pfx.bad_path_re, bestrt):
        is_good_path = 0
      else:
        is_good_path = 1
      if is_good_path != -1 and prev_is_good_path != is_good_path:
        if not DEBUG_EYE:
          self.db.dbQueue.put(("route", (self.pfx.alarm_id, bgp_res_time,
            self.eye_id, is_good_path, self.pfx.bgptime)))
        prev_is_good_path = is_good_path 
      if DEBUG_EYE:
        print >> sys.stderr, "\n>>>>>>>>>> bgp over [nlp]"
        nlp += 1
        print >> sys.stderr, nlp
        if nlp >= 2:
          return
        print >> sys.stderr, "\n<<<<<<<<<<"
      time.sleep(LG_SLEEP_SECONDS)

  def parse_gbxslg_ping(self):
    presp = -1
    ptime = -1
    # session
    ping_req = self.req_session
    urlp = self.addr.split('&')
    urlparams = urlp[1] + self.ping_cmd + self.pfx.liveip
    data = self.get_url_param(urlparams)
    if DEBUG_EYE:
      print >> sys.stderr, "\n>>>>>>>>>> begin ping [urlp]"
      print >> sys.stderr, urlp[0], urlp[1], urlparams
      nlp = 0 
      print >> sys.stderr, "\n<<<<<<<<<<"
    # run
    while True:
      if time.time() > self.deadline:
        return
      resp_str = ''
      beg = time.time()
      post_resp = ping_req.post(urlp[0], data=data,
          timeout=self.ping_timeout*5+15)
      if post_resp and post_resp.text:
        ping_match = re.match(
            '.+<PRE>.+\n([!\S]{5})\n.+</PRE>', post_resp.text, re.DOTALL)
        if ping_match:
          resp_str = ping_match.group(1)
        else:
          return
      else:
        return
      end = time.time()
      if DEBUG_EYE:
        print >> sys.stderr, "\n>>>>>>>>>> get ping output [resp_str]"
        print >> sys.stderr, resp_str
        print >> sys.stderr, "\n---------- [beg, end]"
        print >> sys.stderr, beg, end
        print >> sys.stderr, "\n<<<<<<<<<<"
      time_resp = self.get_ping_res_dot_mark(resp_str, beg, end)
      for ctime in xrange(int(beg), int(end)+1):
        if ctime in time_resp:
          resp = time_resp[ctime]
          if (presp != resp) and (ptime != ctime):
            if not DEBUG_EYE:
              self.db.dbQueue.put(("probe", (self.pfx.alarm_id, ctime,
                self.eye_id, resp, self.pfx.bgptime)))
            presp = resp
            ptime = ctime
      if DEBUG_EYE:
        print >> sys.stderr, "\n>>>>>>>>>> get ping result [time_resp]"
        print >> sys.stderr, time_resp
        print >> sys.stderr, "\n---------- [nlp]"
        nlp += 1
        print >> sys.stderr, nlp
        if nlp >= 2:
          return
        print >> sys.stderr, "\n<<<<<<<<<<"
      time.sleep(LG_SLEEP_SECONDS)

  def parse_savvislg_bgp(self):
    prev_is_good_path = -1
    # session
    bgp_req = self.req_session
    urlp = self.addr.split('&')
    if self.bgpip:
      urlparams = urlp[1] + self.bgp_cmd + self.pfx.liveip
    else:
      urlparams = urlp[1] + self.bgp_cmd + self.pfx.dec
    data = self.get_url_param(urlparams)
    if DEBUG_EYE:
      print >> sys.stderr, "\n>>>>>>>>>> begin bgp [urlp]"
      print >> sys.stderr, urlp[0], urlp[1], urlparams
      nlp = 0 
      print >> sys.stderr, "\n<<<<<<<<<<"
    # run
    while True:
      if time.time() > self.deadline:
        return
      bestrt = ''
      beg = time.time()
      post_resp = bgp_req.post(urlp[0], data=data, timeout=30)
      if post_resp and post_resp.text:
        bestrt_match = re.match(
            '.+<PRE>.*(\nBGP.+)</PRE>',
            post_resp.text, re.DOTALL)
        if bestrt_match:
          reslines = route_match.group(1).split('\n')
          bestrt = self.get_cisco_best_aspath(reslines)
        else:
          time.sleep(LG_SLEEP_SECONDS)
          continue
      else:
        return
      end = time.time()
      bgp_res_time = int((beg + end) / 2)
      if DEBUG_EYE:
        print >> sys.stderr, "\n>>>>>>>>>> get bgp output [bestrt]"
        print >> sys.stderr, bestrt
        print >> sys.stderr, "\n---------- [html]"
        print >> sys.stderr, post_resp.text
        print >> sys.stderr, "\n---------- [beg, end, bgp_res_time]"
        print >> sys.stderr, beg, end, bgp_res_time
        print >> sys.stderr, "\n<<<<<<<<<<"
      if not bestrt:
        time.sleep(LG_SLEEP_SECONDS)
        continue
      if re.search(self.pfx.bad_path_re, bestrt):
        is_good_path = 0
      else:
        is_good_path = 1
      if is_good_path != -1 and prev_is_good_path != is_good_path:
        if not DEBUG_EYE:
          self.db.dbQueue.put(("route", (self.pfx.alarm_id, bgp_res_time,
            self.eye_id, is_good_path, self.pfx.bgptime)))
        prev_is_good_path = is_good_path 
      if DEBUG_EYE:
        print >> sys.stderr, "\n>>>>>>>>>> bgp over [nlp]"
        nlp += 1
        print >> sys.stderr, nlp
        if nlp >= 2:
          return
        print >> sys.stderr, "\n<<<<<<<<<<"
      time.sleep(LG_SLEEP_SECONDS)

  def parse_savvislg_ping(self):
    presp = -1
    ptime = -1
    # session
    ping_req = self.req_session
    urlp = self.addr.split('&')
    urlparams = urlp[1] + self.ping_cmd + self.pfx.liveip
    data = self.get_url_param(urlparams)
    if DEBUG_EYE:
      print >> sys.stderr, "\n>>>>>>>>>> begin ping [urlp]"
      print >> sys.stderr, urlp[0], urlp[1], urlparams
      nlp = 0 
      print >> sys.stderr, "\n<<<<<<<<<<"
    # run
    while True:
      if time.time() > self.deadline:
        return
      resp = -1
      beg = time.time()
      post_resp = ping_req.post(urlp[0], data=data,
          timeout=self.ping_timeout+15)
      if post_resp and post_resp.text:
        ping_match = re.match(
            '.+1 targets.+([01]) alive', post_resp.text, re.DOTALL)
        if ping_match:
          if '1' == ping_match.group(1):
            resp = 1
          else:
            resp = 0
        else:
          return
      else:
        return
      end = time.time()
      ctime = int((beg + end) / 2)
      if DEBUG_EYE:
        print >> sys.stderr, "\n>>>>>>>>>> get ping output [resp]"
        print >> sys.stderr, resp
        print >> sys.stderr, "\n---------- [html]"
        print >> sys.stderr, post_resp.text
        print >> sys.stderr, "\n---------- [beg, end, ping_res_time]"
        print >> sys.stderr, beg, end, ctime
        print >> sys.stderr, "\n<<<<<<<<<<"
      if resp != -1 and (presp != resp) and (ptime != ctime):
        if not DEBUG_EYE:
          self.db.dbQueue.put(("probe", (self.pfx.alarm_id, ctime,
            self.eye_id, resp, self.pfx.bgptime)))
        presp = resp
        ptime = ctime
      if DEBUG_EYE:
        print >> sys.stderr, "\n>>>>>>>>>> get ping result [ctime, resp]"
        print >> sys.stderr, ctime, resp
        print >> sys.stderr, "\n---------- [nlp]"
        nlp += 1
        print >> sys.stderr, nlp
        if nlp >= 2:
          return
        print >> sys.stderr, "\n<<<<<<<<<<"
      time.sleep(LG_SLEEP_SECONDS)

  def parse_bboilg_bgp(self):
    prev_is_good_path = -1
    # session
    bgp_req = self.req_session
    urlp = self.addr.split('&')
    if self.bgpip:
      urlparams = urlp[1] + self.bgp_cmd + self.pfx.liveip
    else:
      urlparams = urlp[1] + self.bgp_cmd + self.pfx.dec
    data = self.get_url_param(urlparams)
    if DEBUG_EYE:
      print >> sys.stderr, "\n>>>>>>>>>> begin bgp [urlp]"
      print >> sys.stderr, urlp[0], urlp[1], urlparams
      nlp = 0 
      print >> sys.stderr, "\n<<<<<<<<<<"
    # run
    while True:
      if time.time() > self.deadline:
        return
      bestrt = ''
      beg = time.time()
      post_resp = None
      post_resp = bgp_req.post(urlp[0], data=data, timeout=30)
      if post_resp and post_resp.text:
        bestrt_match = re.match(
            '.+<pre ?[^>]*>.*(\nBGP.+)</pre>',
            post_resp.text, re.DOTALL | re.IGNORECASE)
        if bestrt_match:
          bgp_res_text = bestrt_match.group(1)
          bgp_res_text = re.sub('<[^<]+>', '', bgp_res_text)
          reslines = bgp_res_text.split('\n')
          bestrt = self.get_cisco_best_aspath(reslines)
        else:
          time.sleep(LG_SLEEP_SECONDS)
          continue
      else:
        return
      end = time.time()
      bgp_res_time = int((beg + end) / 2)
      if DEBUG_EYE:
        print >> sys.stderr, "\n>>>>>>>>>> get bgp output [bestrt]"
        print >> sys.stderr, bestrt
        print >> sys.stderr, "\n---------- [post_resp.text]"
        print >> sys.stderr, '%r' % post_resp.text 
        print >> sys.stderr, "\n---------- [beg, end, bgp_res_time]"
        print >> sys.stderr, beg, end, bgp_res_time
        print >> sys.stderr, "\n<<<<<<<<<<"
      if not bestrt:
        time.sleep(LG_SLEEP_SECONDS)
        continue
      if re.search(self.pfx.bad_path_re, bestrt):
        is_good_path = 0
      else:
        is_good_path = 1
      if is_good_path != -1 and prev_is_good_path != is_good_path:
        if not DEBUG_EYE:
          self.db.dbQueue.put(("route", (self.pfx.alarm_id, bgp_res_time,
            self.eye_id, is_good_path, self.pfx.bgptime)))
        prev_is_good_path = is_good_path 
      if DEBUG_EYE:
        print >> sys.stderr, "\n>>>>>>>>>> bgp over [nlp]"
        nlp += 1
        print >> sys.stderr, nlp
        if nlp >= 2:
          return
        print >> sys.stderr, "\n<<<<<<<<<<"
      time.sleep(LG_SLEEP_SECONDS)

  def parse_bboilg_ping(self):
    presp = -1
    ptime = -1
    # session
    ping_req = self.req_session
    urlp = self.addr.split('&')
    urlparams = urlp[1] + self.ping_cmd + self.pfx.liveip
    data = self.get_url_param(urlparams)
    if DEBUG_EYE:
      print >> sys.stderr, "\n>>>>>>>>>> begin ping [urlp]"
      print >> sys.stderr, urlp[0], urlp[1], urlparams
      nlp = 0 
      print >> sys.stderr, "\n<<<<<<<<<<"
    # run
    while True:
      if time.time() > self.deadline:
        return
      resp = -1
      resp_str = ''
      beg = time.time()
      post_resp = ping_req.post(urlp[0], data=data,
          timeout=self.ping_timeout*5+15)
      if post_resp and post_resp.text:
        ping_match = re.match(
            '.+<pre ?[^>]*>.+\n([!\S]{5})\r?\n.+</pre>', post_resp.text,
            re.DOTALL | re.IGNORECASE)
        if ping_match:
          resp_str = ping_match.group(1)
        else:
          return
      else:
        return
      end = time.time()
      if DEBUG_EYE:
        print >> sys.stderr, "\n>>>>>>>>>> get ping output [resp_str]"
        print >> sys.stderr, resp_str
        print >> sys.stderr, '%r' % post_resp.text[:-200]
        print >> sys.stderr, "\n---------- [beg, end]"
        print >> sys.stderr, beg, end
        print >> sys.stderr, "\n<<<<<<<<<<"
      time_resp = self.get_ping_res_dot_mark(resp_str, beg, end)
      for ctime in xrange(int(beg), int(end)+1):
        if ctime in time_resp:
          resp = time_resp[ctime]
          if (presp != resp) and (ptime != ctime):
            if not DEBUG_EYE:
              self.db.dbQueue.put(("probe", (self.pfx.alarm_id, ctime,
                self.eye_id, resp, self.pfx.bgptime)))
            presp = resp
            ptime = ctime
      if DEBUG_EYE:
        print >> sys.stderr, "\n>>>>>>>>>> get ping result [time_resp]"
        print >> sys.stderr, time_resp
        print >> sys.stderr, "\n---------- [nlp]"
        nlp += 1
        print >> sys.stderr, nlp
        if nlp >= 2:
          return
        print >> sys.stderr, "\n<<<<<<<<<<"
      time.sleep(LG_SLEEP_SECONDS)

  def parse_hgclg_bgp(self):
    self.parse_bboilg_bgp()

  def parse_hgclg_ping(self):
    self.parse_bboilg_ping()

  def parse_unwiredlg_bgp(self):
    self.parse_bboilg_bgp()

  def parse_unwiredlg_ping(self):
    self.parse_bboilg_ping()

  def parse_tatalg_bgp(self):
    prev_is_good_path = -1
    # session
    bgp_req = self.req_session
    urlp = self.addr.split('&')
    if self.bgpip:
      urlparams = urlp[1] + self.bgp_cmd + self.pfx.liveip
    else:
      urlparams = urlp[1] + self.bgp_cmd + self.pfx.dec
    data = self.get_url_param(urlparams)
    if DEBUG_EYE:
      print >> sys.stderr, "\n>>>>>>>>>> begin bgp [urlp]"
      print >> sys.stderr, urlp[0], urlp[1], urlparams
      nlp = 0 
      print >> sys.stderr, "\n<<<<<<<<<<"
    # run
    while True:
      if time.time() > self.deadline:
        return
      bestrt = ''
      beg = time.time()
      post_resp = bgp_req.post(urlp[0], data=data, timeout=30)
      if post_resp and post_resp.text:
        bestrt_match = re.match(
            '.+<PRE>.*\nBGP.+\n<FONT COLOR="#008800">  ([^ ][^<]+)</FONT>',
            post_resp.text, re.DOTALL)
        if bestrt_match:
          bestrt = re.sub('[, ]*\([^\(]+\)[, ]*', ' ', bestrt_match.group(1))
          bestrt = re.sub('[, ]*\{[^\{]+\}[, ]*', ' ', bestrt).strip()
          if bestrt.lower() == 'local':
            bestrt = '%d' % self.asn
        else:
          time.sleep(LG_SLEEP_SECONDS)
          continue
      else:
        return
      end = time.time()
      bgp_res_time = int((beg + end) / 2)
      if DEBUG_EYE:
        print >> sys.stderr, "\n>>>>>>>>>> get bgp output [bestrt]"
        print >> sys.stderr, bestrt
        print >> sys.stderr, "\n---------- [beg, end, bgp_res_time]"
        print >> sys.stderr, beg, end, bgp_res_time
        print >> sys.stderr, "\n<<<<<<<<<<"
      if not bestrt:
        is_good_path = -1
      if re.search(self.pfx.bad_path_re, bestrt):
        is_good_path = 0
      else:
        is_good_path = 1
      if is_good_path != -1 and prev_is_good_path != is_good_path:
        if not DEBUG_EYE:
          self.db.dbQueue.put(("route", (self.pfx.alarm_id, bgp_res_time,
            self.eye_id, is_good_path, self.pfx.bgptime)))
        prev_is_good_path = is_good_path 
      if DEBUG_EYE:
        print >> sys.stderr, "\n>>>>>>>>>> bgp over [nlp]"
        nlp += 1
        print >> sys.stderr, nlp
        if nlp >= 2:
          return
        print >> sys.stderr, "\n<<<<<<<<<<"
      time.sleep(LG_SLEEP_SECONDS)

  def parse_tatalg_ping(self):
    self.parse_bboilg_ping()

  def parse_runnetlg_bgp(self):
    self.parse_bboilg_bgp()

  def parse_runnetlg_ping(self):
    self.parse_bboilg_ping()

  def parse_ilanlg_bgp(self):
    prev_is_good_path = -1
    # session
    bgp_req = self.req_session
    if self.bgpip:
      url = self.addr + self.bgp_cmd + self.pfx.liveip + '&src=argus.tsinghua'
    else:
      url = self.addr + self.bgp_cmd + self.pfx.dec + '&src=argus.tsinghua'
    if DEBUG_EYE:
      print >> sys.stderr, "\n>>>>>>>>>> begin bgp [url]"
      print >> sys.stderr, url
      nlp = 0 
      print >> sys.stderr, "\n<<<<<<<<<<"
    # run
    while True:
      if time.time() > self.deadline:
        return
      bestrt = ''
      beg = time.time()
      get_resp = bgp_req.get(url, timeout=30)
      if get_resp and get_resp.text:
        bestrt_match = re.match(
            '.+<pre>.*(\nBGP.+)</pre>',
            get_resp.text, re.DOTALL | re.IGNORECASE)
        if bestrt_match:
          bgp_res_text = bestrt_match.group(1)
          bgp_res_text = re.sub('<[^<]+>', '', bgp_res_text)
          reslines = bgp_res_text.split('\n')
          bestrt = self.get_cisco_best_aspath(reslines)
        else:
          time.sleep(LG_SLEEP_SECONDS)
          continue
      else:
        return
      end = time.time()
      bgp_res_time = int((beg + end) / 2)
      if DEBUG_EYE:
        print >> sys.stderr, "\n>>>>>>>>>> get bgp output [bestrt]"
        print >> sys.stderr, bestrt
        print >> sys.stderr, "\n---------- [beg, end, bgp_res_time]"
        print >> sys.stderr, beg, end, bgp_res_time
        print >> sys.stderr, "\n<<<<<<<<<<"
      if not bestrt:
        time.sleep(LG_SLEEP_SECONDS)
        continue
      if re.search(self.pfx.bad_path_re, bestrt):
        is_good_path = 0
      else:
        is_good_path = 1
      if is_good_path != -1 and prev_is_good_path != is_good_path:
        if not DEBUG_EYE:
          self.db.dbQueue.put(("route", (self.pfx.alarm_id, bgp_res_time,
            self.eye_id, is_good_path, self.pfx.bgptime)))
        prev_is_good_path = is_good_path 
      if DEBUG_EYE:
        print >> sys.stderr, "\n>>>>>>>>>> bgp over [nlp]"
        nlp += 1
        print >> sys.stderr, nlp
        if nlp >= 2:
          return
        print >> sys.stderr, "\n<<<<<<<<<<"
      time.sleep(LG_SLEEP_SECONDS)

  def parse_ilanlg_ping(self):
    presp = -1
    ptime = -1
    # session
    ping_req = self.req_session
    url = self.addr + self.ping_cmd + self.pfx.liveip + '&src=argus.tsinghua'
    if DEBUG_EYE:
      print >> sys.stderr, "\n>>>>>>>>>> begin ping [url]"
      print >> sys.stderr, url
      nlp = 0 
      print >> sys.stderr, "\n<<<<<<<<<<"
    # run
    while True:
      if time.time() > self.deadline:
        return
      resp = -1
      beg = time.time()
      get_resp = ping_req.get(url, timeout=self.ping_timeout*5+15)
      if get_resp and get_resp.text:
        ping_match = re.match(
            '.+<pre>.+\n([!\S]{5})\r?\n.+</pre>', get_resp.text,
            re.DOTALL | re.IGNORECASE)
        if ping_match:
          resp_str = ping_match.group(1)
        else:
          return
      else:
        return
      end = time.time()
      if DEBUG_EYE:
        print >> sys.stderr, "\n>>>>>>>>>> get ping output [resp]"
        print >> sys.stderr, resp_str
        print >> sys.stderr, "\n---------- [beg, end]"
        print >> sys.stderr, beg, end
        print >> sys.stderr, "\n<<<<<<<<<<"
      time_resp = self.get_ping_res_dot_mark(resp_str, beg, end)
      for ctime in xrange(int(beg), int(end)+1):
        if ctime in time_resp:
          resp = time_resp[ctime]
          if (presp != resp) and (ptime != ctime):
            if not DEBUG_EYE:
              self.db.dbQueue.put(("probe", (self.pfx.alarm_id, ctime,
                self.eye_id, resp, self.pfx.bgptime)))
            presp = resp
            ptime = ctime
      if DEBUG_EYE:
        print >> sys.stderr, "\n>>>>>>>>>> get ping result [time_resp]"
        print >> sys.stderr, time_resp
        nlp += 1
        print >> sys.stderr, nlp
        if nlp >= 2:
          return
        print >> sys.stderr, "\n<<<<<<<<<<"
      time.sleep(LG_SLEEP_SECONDS)

  def parse_cernlg_bgp(self):
    self.parse_bboilg_bgp()

  def parse_cernlg_ping(self):
    self.parse_bboilg_ping()

  def parse_switchlg_bgp(self):
    self.parse_bboilg_bgp()

  def parse_switchlg_ping(self):
    self.parse_bboilg_ping()

  def parse_morenetlg_bgp(self):
    self.parse_bboilg_bgp()

  def parse_morenetlg_ping(self):
    self.parse_bboilg_ping()

  def parse_xocomlg_bgp(self):
    prev_is_good_path = -1
    # session
    bgp_req = self.req_session
    urlp = self.addr.split('&')
    if self.bgpip:
      urlparams = urlp[1] + self.bgp_cmd + self.pfx.liveip
    else:
      urlparams = urlp[1] + self.bgp_cmd + self.pfx.dec
    data = self.get_url_param(urlparams)
    if DEBUG_EYE:
      print >> sys.stderr, "\n>>>>>>>>>> begin bgp [urlp]"
      print >> sys.stderr, urlp[0], urlp[1], urlparams
      nlp = 0 
      print >> sys.stderr, "\n<<<<<<<<<<"
    # run
    while True:
      if time.time() > self.deadline:
        return
      bestrt = ''
      beg = time.time()
      post_resp = None
      post_resp = bgp_req.post(urlp[0], data=data, timeout=30)
      if post_resp and post_resp.text:
        bestrt_match = re.match(
            '.+<pre>.*\n BGP.+\n \* \* \*  ([l\(1-9][^\n]*)\n.+</pre>',
            post_resp.text, re.DOTALL | re.IGNORECASE)
        if bestrt_match:
          bestrt = re.sub('[, ]*\([^\(]+\)[, ]*', ' ', bestrt_match.group(1))
          bestrt = re.sub('[, ]*\{[^\{]+\}[, ]*', ' ', bestrt).strip()
          if bestrt.lower() == 'local':
            bestrt = '%d' % self.asn
        else:
          time.sleep(LG_SLEEP_SECONDS)
          continue
      else:
        return
      end = time.time()
      bgp_res_time = int((beg + end) / 2)
      if DEBUG_EYE:
        print >> sys.stderr, "\n>>>>>>>>>> get bgp output [bestrt]"
        print >> sys.stderr, bestrt
        #print >> sys.stderr, "\n---------- [post_resp.text]"
        #print >> sys.stderr, '%r' % post_resp.text[1000:] 
        print >> sys.stderr, "\n---------- [beg, end, bgp_res_time]"
        print >> sys.stderr, beg, end, bgp_res_time
        print >> sys.stderr, "\n<<<<<<<<<<"
      if not bestrt:
        time.sleep(LG_SLEEP_SECONDS)
        continue
      if re.search(self.pfx.bad_path_re, bestrt):
        is_good_path = 0
      else:
        is_good_path = 1
      if is_good_path != -1 and prev_is_good_path != is_good_path:
        if not DEBUG_EYE:
          self.db.dbQueue.put(("route", (self.pfx.alarm_id, bgp_res_time,
            self.eye_id, is_good_path, self.pfx.bgptime)))
        prev_is_good_path = is_good_path 
      if DEBUG_EYE:
        print >> sys.stderr, "\n>>>>>>>>>> bgp over [nlp]"
        nlp += 1
        print >> sys.stderr, nlp
        if nlp >= 2:
          return
        print >> sys.stderr, "\n<<<<<<<<<<"
      time.sleep(LG_SLEEP_SECONDS)

  def parse_xocomlg_ping(self):
    presp = -1
    ptime = -1
    # session
    ping_req = self.req_session
    urlp = self.addr.split('&')
    urlparams = urlp[1] + self.ping_cmd + self.pfx.liveip
    data = self.get_url_param(urlparams)
    if DEBUG_EYE:
      print >> sys.stderr, "\n>>>>>>>>>> begin ping [urlp]"
      print >> sys.stderr, urlp[0], urlp[1], urlparams
      nlp = 0 
      print >> sys.stderr, "\n<<<<<<<<<<"
    # run
    while True:
      if time.time() > self.deadline:
        return
      resp = -1
      resp_str = ''
      beg = time.time()
      post_resp = ping_req.post(urlp[0], data=data,
          timeout=self.ping_timeout*5+15)
      if post_resp and post_resp.text:
        ping_match = re.match(
            '.+<pre>.+\n ([!\S]{5})\r?\n.+</pre>', post_resp.text,
            re.DOTALL | re.IGNORECASE)
        if ping_match:
          resp_str = ping_match.group(1)
        else:
          return
      else:
        return
      end = time.time()
      if DEBUG_EYE:
        print >> sys.stderr, "\n>>>>>>>>>> get ping output [resp_str]"
        print >> sys.stderr, resp_str
        #print >> sys.stderr, '%r' % post_resp.text[1000:]
        print >> sys.stderr, "\n---------- [beg, end]"
        print >> sys.stderr, beg, end
        print >> sys.stderr, "\n<<<<<<<<<<"
      time_resp = self.get_ping_res_dot_mark(resp_str, beg, end)
      for ctime in xrange(int(beg), int(end)+1):
        if ctime in time_resp:
          resp = time_resp[ctime]
          if (presp != resp) and (ptime != ctime):
            if not DEBUG_EYE:
              self.db.dbQueue.put(("probe", (self.pfx.alarm_id, ctime,
                self.eye_id, resp, self.pfx.bgptime)))
            presp = resp
            ptime = ctime
      if DEBUG_EYE:
        print >> sys.stderr, "\n>>>>>>>>>> get ping result [time_resp]"
        print >> sys.stderr, time_resp
        print >> sys.stderr, "\n---------- [nlp]"
        nlp += 1
        print >> sys.stderr, nlp
        if nlp >= 2:
          return
        print >> sys.stderr, "\n<<<<<<<<<<"
      time.sleep(LG_SLEEP_SECONDS)

  def parse_demoslg_bgp(self):
    self.parse_bboilg_bgp()

  def parse_demoslg_ping(self):
    self.parse_bboilg_ping()

  def parse_iinetlg_bgp(self):
    self.parse_bboilg_bgp()

  def parse_iinetlg_ping(self):
    self.parse_bboilg_ping()

  def parse_dfnlg_bgp(self):
    self.parse_gbxslg_bgp()

  def parse_dfnlg_ping(self):
    self.parse_bboilg_ping()

  def parse_rogerslg_bgp(self):
    prev_is_good_path = -1
    # session
    bgp_req = self.req_session
    urlp = self.addr.split('&')
    if self.bgpip:
      urlparams = urlp[1] + self.bgp_cmd + self.pfx.liveip
    else:
      urlparams = urlp[1] + self.bgp_cmd + self.pfx.dec
    data = self.get_url_param(urlparams)
    if DEBUG_EYE:
      print >> sys.stderr, "\n>>>>>>>>>> begin bgp [urlp]"
      print >> sys.stderr, urlp[0], urlp[1], urlparams
      nlp = 0 
      print >> sys.stderr, "\n<<<<<<<<<<"
    # run
    while True:
      if time.time() > self.deadline:
        return
      bestrt = ''
      beg = time.time()
      post_resp = bgp_req.post(urlp[0], data=data, timeout=30)
      if post_resp and post_resp.text:
        bestrt_match = re.match(
            '.+<PRE>.*\nBGP.+\n<FONT COLOR="#FF0000">  ([^ ][^\n]+)</FONT>',
            post_resp.text, re.DOTALL)
        if bestrt_match:
          bestrt = re.sub('<[^<]+>', '', bestrt_match.group(1))
          bestrt = re.sub('[, ]*\([^\(]+\)[, ]*', ' ', bestrt)
          bestrt = re.sub('[, ]*\{[^\{]+\}[, ]*', ' ', bestrt).strip()
          if bestrt.lower() == 'local':
            bestrt = '%d' % self.asn
        else:
          time.sleep(LG_SLEEP_SECONDS)
          continue
      else:
        return
      end = time.time()
      bgp_res_time = int((beg + end) / 2)
      if DEBUG_EYE:
        print >> sys.stderr, "\n>>>>>>>>>> get bgp output [bestrt]"
        print >> sys.stderr, bestrt
        #print >> sys.stderr, "\n---------- [html]"
        #print >> sys.stderr, post_resp.text
        print >> sys.stderr, "\n---------- [beg, end, bgp_res_time]"
        print >> sys.stderr, beg, end, bgp_res_time
        print >> sys.stderr, "\n<<<<<<<<<<"
      if not bestrt:
        is_good_path = -1
      if re.search(self.pfx.bad_path_re, bestrt):
        is_good_path = 0
      else:
        is_good_path = 1
      if is_good_path != -1 and prev_is_good_path != is_good_path:
        if not DEBUG_EYE:
          self.db.dbQueue.put(("route", (self.pfx.alarm_id, bgp_res_time,
            self.eye_id, is_good_path, self.pfx.bgptime)))
        prev_is_good_path = is_good_path 
      if DEBUG_EYE:
        print >> sys.stderr, "\n>>>>>>>>>> bgp over [nlp]"
        nlp += 1
        print >> sys.stderr, nlp
        if nlp >= 2:
          return
        print >> sys.stderr, "\n<<<<<<<<<<"
      time.sleep(LG_SLEEP_SECONDS)

  def parse_rogerslg_ping(self):
    self.parse_bboilg_ping()

  def run_looking_glass(self):
    try:
      if self.cmd_type == 'bgp': # show bgp
        bgp_parser = getattr(self, self.parse_bgp)
        bgp_parser()
      else: # ping liveip
        ping_parser = getattr(self, self.parse_ping)
        addrutil.log(int(time.time()), 0, "Y:parse ping function:%s" % self.parse_ping)
        ping_parser()
      # if no result, take a break, for about one hour
      ctime = int(time.time())
      if ctime < self.deadline - 30:
        no_result_eye.put(self.eye_id)
      addrutil.log(int(time.time()), 0, "Y:ping has no exception!")
    except Exception as e:
      addrutil.log(int(time.time()), 0,
        "eye#%d AS-%d %r terminated." % (self.eye_id, self.asn, e))
      # if timeout, take a break, for about one hour
      timeout_eye.put(self.eye_id)
    finally:
      if self.req_session is not None:
        return
        #self.req_session.close() #index by yy
        #del self.req_session # commented, to reuse session
  
  def run_route_server(self):
    tel = None
    try:
      tel = telnetlib.Telnet(self.addr, timeout=30)
      user_promt = ''
      user_input = ''
      pwd_promt = ''
      pwd_input = ''
      if self.user:
        user_promt, user_input = self.user.split('/')
        user_promt = '\n' + user_promt
      if self.pwd:
        pwd_promt, pwd_input = self.pwd.split('/')
        pwd_promt = pwd_promt
      # login
      while True:
        expected = -1 # -1: no, 0: user, 1: pwd, 2: promt
        if self.user:
          if self.pwd:
            exp_res = tel.expect([user_promt, pwd_promt, self.promt], 30)
            expected = exp_res[0]
          else:
            exp_res = tel.expect([user_promt, self.promt], 30)
            if exp_res[0] == 0:
              expected = 0
            elif exp_res[0] == 1:
              expected = 2
        else:
          if self.pwd:
            exp_res = tel.expect([pwd_promt, self.promt], 30)
            if exp_res[0] == 0:
              expected = 1
            elif exp_res[0] == 1:
              expected = 2
          else:
            exp_res = tel.expect([self.promt, ], 30)
            if exp_res[0] == 0:
              expected = 2
        if DEBUG_EYE:
          print >> sys.stderr, "\n>>>>>>>>>> login wait stable [exp_res]"
          print >> sys.stderr, exp_res
          print >> sys.stderr, "\n---------- [expected]"
          print >> sys.stderr, expected
          print >> sys.stderr, "\n<<<<<<<<<<"
        if expected == 0:
          tel.write('%s\n' % user_input)
        elif expected == 1:
          tel.write('%s\n' % pwd_input)
        elif expected == 2:
          tel.write('\n')
          break
        else: # -1
          if tel:
            tel.close()
          return
        # timeout
        if time.time() > self.deadline:
          if tel:
            tel.close()
          return
      if self.cmd_type == 'bgp': # show bgp
        bgp_parser = getattr(self, self.parse_bgp)
        bgp_parser(tel)
      else: # ping liveip
        ping_parser = getattr(self, self.parse_ping)
        addrutil.log(int(time.time()), 0, "Y:parse ping function:%s" % self.parse_ping)
        ping_parser(tel)
      # if no result, take a break, for about one hour
      ctime = int(time.time())
      if ctime < self.deadline - 30:
        no_result_eye.put(self.eye_id)
    except Exception as e:
      addrutil.log(int(time.time()), 0,
        "eye#%d AS-%d %r terminated." % (self.eye_id, self.asn, e))
      #traceback.print_exc()
      # if timeout, take a break, for about one hour
      timeout_eye.put(self.eye_id)
    finally:
      if tel:
        tel.close()

session_pool = {} # url <--> sessions
def get_session_from_pool(url):
  global session_pool
  url_sess = session_pool.get(url, None)
  if url_sess:
    for s in url_sess:
      if s[1] == 0:
        s[1] = 1
        return s[0]
    sess = requests.session(prefetch=True)
    session_pool[url].append([sess, 1])
    return sess
  else:
    sess = requests.session()
    session_pool[url] = [[sess, 1], ]
    return sess
  #return requests.session(prefetch=True)

def release_session_pool():
  global session_pool
  for k, v in session_pool.iteritems():
    for s in v:
      s[1] = 0
  #return

targetQueue = Queue.PriorityQueue()
def probeTargetRun():
  #prevprob = 0
  addrutil.log(int(time.time()), 0, "Y:probe running!")
  while True:
    priority, pfx, inq_time, db = targetQueue.get(True)
    outQTime = int(time.time())
    if outQTime - pfx.bgptime > 60:
      continue
    #
    #if pfx.bgptime - prevprob < 150:
    #  continue
    #prevprob = pfx.bgptime 
    #
    ## reduce # of anomalies
    #if pfx.is_live == 0 and pfx.bad_path_re.count(' ') == 2:
    #  continue
    #
    if pfx.is_live == 0 and targetQueue.qsize() > 0:
      continue
    addrutil.log(outQTime, 0,
      "INFO/RIO: alarm %d/%d OUT targetQueue. pathre [%s]" % (
      pfx.alarm_id, pfx.bgptime, pfx.bad_path_re))
    threads = []
    eye = db.get_eye(pfx.is_ipv6, True)
    addrutil.log(int(time.time()), 0, "Y:there are %d eyes" % len(eye))
    for e in eye:
      if e.get('type') == 'l':
        url = e.get('addr')
        url = url[ : url.index('?')]
        threads.append(ProbeTargetThread(e, db, 'bgp', pfx,
          get_session_from_pool('bgp_' + url)))
        threads.append(ProbeTargetThread(e, db, 'ip', pfx,
          get_session_from_pool('ip__' + url)))
      else:
        threads.append(ProbeTargetThread(e, db, 'bgp', pfx))
        threads.append(ProbeTargetThread(e, db, 'ip', pfx))
    for t in threads:
      t.start()
    for t in threads:
      t.join(pfx.bgptime + 130 - int(time.time())) # TODO: wait 130 seconds
    db.dbQueue.put(("signature", (pfx.bgptime, pfx.alarm_id)))
    release_session_pool()
    addrutil.log(int(time.time()), 0,
      "INFO/RIOE: alarm %d/%d OVER." % (pfx.alarm_id, pfx.bgptime))
    # eyes failed can take a break for about one hour
    fcid = set()
    try:
      while not timeout_eye.empty():
        fc = timeout_eye.get(False)
        fcid.add('%d' % fc)
    except:
      pass
    feid = set()
    try:
      while not no_result_eye.empty():
        fe = no_result_eye.get(False)
        feid.add('%d' % fe)
    except:
      pass
    if feid or fcid:
      db.update_failed_eye_counter(feid, fcid)

def probeTarget(pfx, db):
  inq_time = int(time.time())
  if inq_time - pfx.bgptime <= 60:
    addrutil.log(inq_time, 0,
      "INFO/RI: alarm %d/%d IN targetQueue. pfx %s ip %s live %d. qsize: %d" % (
      pfx.alarm_id, pfx.bgptime, pfx.dec, pfx.liveip, pfx.is_live, targetQueue.qsize()))
    # OA > AA > PA, then latest first
    priority = (pfx.bad_path_re.count(' '), -pfx.bgptime, pfx.alarm_id)
    targetQueue.put((priority, pfx, inq_time, db))

################################################
# Followings for debug only
################################################



if __name__ == '__main__':
  db = dbmgr.DBMgr()
  # TODO: remember to modify `counter_id` in db
  #add_rogers_lg(db)
  #test_eye(db)
  db.close()

